# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter

import pymongo
import math
from hronet.items import HronetHotHTMLItem
from hronet.items import HronetHotMainPircture
import requests
import base64

class HronetPipeline:
    
    def __init__(self):
        self.myclient = pymongo.MongoClient("mongodb://localhost:27017/")
        if "hronet" not in self.myclient.list_database_names():
            raise Exception("不存在[hronet]数据库")
        self.mydb = self.myclient["hronet"]
        if "hronet_hot" not in self.mydb.list_collection_names():
            raise Exception("不存在[hronet_hot]集合")
        self.hronetHot_col = self.mydb["hronet_hot"]
        if "hronet_html" not in self.mydb.list_collection_names():
            raise Exception("不存在[hronet_html]集合")
        self.hronetHTML_col = self.mydb["hronet_html"]
        if "hronet_main_picture" not in self.mydb.list_collection_names():
            raise Exception("不存在[hronet_main_picture]集合")
        self.hronet_main_picture = self.mydb["hronet_main_picture"]
        
    
    # 爬虫开始 就会调用
    def open_spider(self,spider):
        print('爬虫开始')
    
    def process_item(self, item, spider):
        if not isDuplicateData(self, item):
            if not isAvailableData(item):
                item['isDeleted'] = 1
            html = item['html']
            htmlSize = len(html) if html is not None else 0
            size = htmlSize / (1024 * 1024) 
            hronetHotItem = self.hronetHot_col.insert_one({'travelDay': item['travelDay'], 'author': item['author'], 'type': item['type'], 'title': item['title'], 'size': size, 'sendTime': item['sendTime'], 'createTime': item['createTime'], 'isDeleted': item['isDeleted']})
            parseBase64InsertMogoDB(self, item['mainPictureUrl'], hronetHotItem.inserted_id)
            # 12MB 每个HTML分块
            maxSize = 1024 * 1024 * 12
            part = math.ceil(htmlSize / maxSize)
            index = 0
            for i in range(1, part+1):
                splitHtml = ''
                if i * maxSize <= len(html):
                    splitHtml = html[index:i*maxSize]
                else:
                    splitHtml = html[index:]
                index = i * maxSize
                hronetHtmlItem = HronetHotHTMLItem(hronethotId = hronetHotItem.inserted_id, html = splitHtml, sequence = i, isDeleted = 0)
                self.hronetHTML_col.insert_one(hronetHtmlItem)
            return hronetHotItem
        else:
            title = item['title'] if item['title'] is not None else 'null'
            print('重复【'+ title +'】')
            return item
    
    # 爬虫关闭 就会调用
    def close_spider(self,spider):
        # 关闭连接
        self.myclient.close()
        print('爬虫结束')

#是否是有效数据
def isAvailableData(item):
    if item['travelDay'] is None:
        return False
    if item['author'] is None:
        return False
    if item['type'] is None:
        return False
    if item['title'] is None:
        return False
    if item['html'] is None:
        return False
    if item['sendTime'] is None:
        return False
    return True

#是否是重复数据
def isDuplicateData(self, item):
    vertifyDuplicateFlag = 0
    if item['travelDay'] is not None and self.hronetHot_col.find({'travelDay': item['travelDay']}).count() > 0:
        vertifyDuplicateFlag = vertifyDuplicateFlag + 1
    if item['author'] is not None and self.hronetHot_col.find({'author': item['author']}).count() > 0:
        vertifyDuplicateFlag = vertifyDuplicateFlag + 1
    if item['type'] is not None and self.hronetHot_col.find({'type': item['type']}).count() > 0:
        vertifyDuplicateFlag = vertifyDuplicateFlag + 1
    if item['title'] is not None and self.hronetHot_col.find({'title': item['title']}).count() > 0:
        vertifyDuplicateFlag = vertifyDuplicateFlag + 1
    if item['sendTime'] is not None and self.hronetHot_col.find({'sendTime': item['sendTime']}).count() > 0:
        vertifyDuplicateFlag = vertifyDuplicateFlag + 1
    if vertifyDuplicateFlag == 5:
        return True
    return False

# 将图片转换成base64 分割存储在mogodb
def parseBase64InsertMogoDB(self, imgUrl, articleId):
    dataType = 'data:image/jpeg;base64,'
    if 'jpeg' in imgUrl:
        dataType = 'data:image/jpeg;base64,'
    if 'png' in imgUrl:
        dataType = 'data:image/png;base64,'
    res = requests.get(imgUrl, stream=True)
    if res.status_code == 200:
        base64_data = base64.b64encode(res.content)
        content = dataType + str(base64_data, encoding = "utf-8")
        # 12MB 每个HTML分块
        maxSize = 1024 * 1024 * 12
        part = math.ceil(len(content) / maxSize)
        index = 0
        for i in range(1, part+1):
            splitContent = ''
            if i * maxSize <= len(content):
                splitContent = content[index:i*maxSize]
            else:
                splitContent = content[index:]
            index = i * maxSize
            item = HronetHotMainPircture(hronethotId = articleId, content=splitContent, sequence = i)
            self.hronet_main_picture.insert_one(item)